﻿// YamlScanner.cs
//
// Copyright 2008-2011 by Burt Harris
// Licensed under the Microsoft Public License (Ms-PL)
// See: http://www.codeplex.com/yaml/license
//

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Diagnostics.Contracts;
using System.Globalization;
using Microsoft.Scripting;

namespace YaTools.Yaml
{
    /// <summary>
    /// A hand-generated scanner for YAML colorizing & parsing
    /// </summary>
    [CLSCompliant(false)]
    [ContractVerification(false)] // TODO Remove this whe static verification issues addressed
    public abstract class AbstractYamlScanner : Scanner
    {
        private const int SimpleKeyLengthLimit = 1000;

        public static readonly TraceSwitch TraceTokens =
            new TraceSwitch("YamlScanner.TraceTokens",
                            "Trace YAML scanner raw tokens", "Error");

        public static readonly TraceSwitch TraceStateChanges =
            new TraceSwitch("YamlScanner.TraceStateChanges",
                            "Trace YAML scanner state changes", "Error");

        private static readonly char[] SingleQuoteBreakCharacters = new char[]
                                                                        {
                                                                            Unicode.SingleQuote, Unicode.CR, Unicode.LF,
                                                                            Unicode.EOS
                                                                        };

        private static readonly char[] DoubleQuoteBreakCharacters = new char[]
                                                                        {
                                                                            Unicode.DoubleQuote, Unicode.Backslash,
                                                                            Unicode.CR, Unicode.LF, Unicode.EOS
                                                                        };

        private static readonly char[] BlockPlainBreakCharacters = new char[]
                                                                       {
                                                                           '#', ':', Unicode.Tab, Unicode.CR, Unicode.LF
                                                                           , Unicode.EOS
                                                                       };

        private static readonly char[] FlowPlainBreakCharacters = new[]
                                                                      {
                                                                          ',', '[', ']', '{', '}',
                                                                          // Extra flow indicators
                                                                          '#', ':', Unicode.Tab, Unicode.CR, Unicode.LF,
                                                                          Unicode.EOS
                                                                      };

        private char _chomping;
        private bool _incrementalMode; // Indicates if scanning sink in use
        private Token _lastToken = Token.Newline;
        private Token _token = Token.Newline;

        internal AbstractYamlScanner()
        {
            State = ScanState.Default;
        }

        protected abstract int StateDepth { get; }

        // Note: comments with numbers in brackets refer to the 
        //  syntax production numbers in the YAML 1.1 (not 1.2) document.

        // Note: all intra-line state must go into the ScanState class

        protected ScanState State { get; set; }

        public bool InKey
        {
            get { return State.InKey; }
            private set { State = new ScanState(State.Expect, State.RawIndent, State.StartLocation, value); }
        }

        public bool IsIndentPending
        {
            get { return State.IsIndentPending; }
        }

        /// <summary>Latest scanned Token code</summary>
        public Token Token
        {
            get { return _token; }
        }

        public char Chomping
        {
            get { return _chomping; }
        }

        protected abstract void PushState(ScanState state);
        protected abstract ScanState PopState();

        protected SourceLocation SeekState(SourceLocation location, string text)
        {
            FillBuffer(text, location);
            _incrementalMode = true;
            _token = Token.Newline;
            _lastToken = Token.Newline;
            return location;
        }


        /// <summary>Processes line start tokens, indent/dedent, ignored whitespace, etc.</summary>
        /// <returns>true if a menaingful token is returned, false otherwise</returns>
        private bool ScanIndent()
        {
            if (Peek() == Unicode.EOS)
            {
                if (!_incrementalMode && StateDepth > 0)
                    return PopAndSetToken(Token.Outdent);
                return SetToken(Token.EndOfStream);
            }

            if (ScanEOL())
                return true;

            switch (_lastToken)
            {
                default:
                    return false; // Only have work to do at BOL

                case Token.IndentSpaces:
                    if (IsIndentPending && State.Expect == YamlExpect.BlockNode)
                    {
                        int leading = SkipSpaces();
                        if (leading > 0)
                        {
                            if (!IsListIndicator()
                                && !IsImplicitKey(BlockPlainBreakCharacters)
                                & Peek() != '?')
                            {
                                Backup(leading);
                                return false;
                            }
                            return SetToken(Token.IndentSpaces);
                        }
                    }
                    return false;

                case Token.Newline:
                case Token.Outdent:
                case Token.PlainEnd:

                    if (CurrentIndent != 0)
                        return false;

                    int leadingSpaces = SkipSpaces();

                    // Document-level markup
                    if (leadingSpaces == 0 && ScanDocumentIndicators())
                        return true;

                    char ch = Peek();

                    if (State.Expect == YamlExpect.Literal || State.Expect == YamlExpect.FoldedText)
                    {
                        if (!State.IsIndentPending && leadingSpaces > State.Indent)
                        {
                            Backup(leadingSpaces - State.Indent);
                            return SetToken(Token.IndentSpaces);
                        }
                        else if (IsBreak(ch))
                        {
                            return SetToken(Token.EmptyLine);
                        }
                    }
                    else
                    {
                        if (IsBreak(ch) || IsComment(ch))
                            return ScanCommentOrBreak();
                    }

                    //if ( State.IsIndentPending && leadingSpaces >= State.Indent )
                    //{
                    //    SetIndent( leadingSpaces );
                    //}

                    int extraSpaces = leadingSpaces - State.Indent;

                    if (extraSpaces < 0)
                    {
                        if (ch == Unicode.Tab)
                        {
                            while (Peek() == Unicode.Tab)
                                Read();
                            AddError("YA104");
                            return SetToken(Token.Unexpected);
                        }
                        if (leadingSpaces > 0) Backup(leadingSpaces);
                        return PopAndSetToken(Token.Outdent);
                    }

                    if (extraSpaces > 0)
                    {
                        Backup(extraSpaces);
                    }

                    if (State.Indent > 0)
                        return SetToken(Token.IndentSpaces);
                    break;
            }
            return false;
        }

        private bool IsComment(char ch)
        {
            if (ch == '#')
                return true;

            if (ch == Unicode.Tab)
            {
                using (SaveOffset())
                {
                    ReadSpacesAndTabs();
                    if (Read('#') || IsBreak(Peek()))
                        return true;
                }
            }
            return false;
        }

        private bool ScanSeperation()
        {
            if (ReadSpacesAndTabs() > 0)
            {
                EndToken();
                BeginScan(); // Skip the whitespace
            }
            if (ScanCommentOrBreak())
                return true;
            return false;
        }

        // Generates a special (<-1) indent value indicating more more than 
        // the ones-complement value of indent is expected.
        private static short MoreIndented(int indent) // For productions with "<n"
        {
            if (indent < 0) // Already in special sink
            {
                if (indent == -1)
                    return (short) indent;
                return (short) (indent - 1);
            }
            return (short) ~(indent + 1);
        }

        private bool IsListIndicator()
        {
            if (State.IsBlockContext)
            {
                using (SaveOffset())
                {
                    return Read('-') && IsWhiteSpace(Read());
                }
            }
            return false;
        }

        /// <summary>
        /// Checks for a matching token, where matching means that the text must
        /// match _plus_ that the following character must be whitespace, or a quotation
        /// mark.  
        /// </summary>
        /// <param name="token">Token text to match</param>
        /// <returns>True if a match found, else false.</returns>
        private bool PeekToken(string token)
        {
            using (SaveOffset())
            {
                if (!Read(token))
                    return false;
                char ch = Peek();
                if (!IsWhiteSpace(ch)
                    && ch != Unicode.SingleQuote
                    && ch != Unicode.DoubleQuote)
                    return false;
            }
            return true;
        }

        protected override void OnLineStart()
        {
            char ch;

            while ((ch = Read()) != Unicode.EOS && !IsBreak(ch))
            {
                if (ch < (char) 0x20 && ch != '\t' && ch != '\r' && ch != '\n')
                    AddError("YA101", String.Format("U+{0}", ((int) ch).ToString("X4")));
            }
            _token = Token.Newline;
        }

        /// <summary>
        /// Writes a line to trace output, substituting unicode number spaces for all spaces.
        /// </summary>
        /// <param name="msg">Message to trace</param>
        /// <remarks>
        /// The Replace operation is useful because Visual Studio's test output window is 
        /// rendered using a proportional-width font, with regular spaces are much smaller 
        /// than digits, disturbing attempts at column line up.
        /// </remarks>
        protected static void TraceLine(string msg)
        {
            Trace.WriteLine(msg.Replace(' ', (char) 0x2007));
        }

        private void TraceState(string action)
        {
            if (TraceStateChanges.Level < TraceLevel.Verbose) return;
            TraceLine(string.Format(CultureInfo.InvariantCulture, "  {0,-3}\t{1}: {2}",
                                    TokenStart.Column, action, State));
        }

        private void TraceToken(string prefix, Token token)
        {
            if (TraceTokens.Level < TraceLevel.Verbose) return;

            TraceLine(string.Format(CultureInfo.InvariantCulture, "  {4,-3}\tReturn: {3}\t{0}\t{1,-16}\t{2}", prefix,
                                    token,
                                    DebugDisplay(TokenText), State, TokenStart.Column));
        }

        public Token NextToken()
        {
            _lastToken = _token; // May need to know this
            _token = Token.NoOp; // Start with a clean slate
            BeginScan(); // Setup in base class

            if (!ContinueScanning())
                throw new InvalidOperationException("ContinueScanning failed to produce a token");

            Debug.Assert(_token != Token.NoOp);
            return _token;
        }

        private bool ScanDocumentIndicators()
        {
            // Check for document-level markup and pop up to document level

            if (IsEndHeaders() || IsEndDocument())
            {
                if (IsImplicitKey(BlockPlainBreakCharacters))
                {
                    char c = Peek();
                    ReadLine();
                    SetToken(c == '-' ? Token.DirectivesEnd : Token.DocumentEnd);
                    AddError("YA302", TokenText);
                }

                if (StateDepth > 0)
                    return PopAndSetToken(Token.Outdent);

                Expect(YamlExpect.Document);

                // Match the actual document-level markup here
                if (Read("---"))
                {
                    return SetToken(Token.DirectivesEnd);
                }

                if (IsEndDocument())
                {
                    Read("...");
                    if (!State.IsBlockContext)
                        return ScanUnexpected();
                    return SetToken(Token.DocumentEnd);
                }
            }
            return false;
        }

        private bool IsEndHeaders()
        {
            using (SaveOffset())
            {
                return Read("---") && IsWhiteSpace(Read());
            }
        }

        private bool IsEndDocument()
        {
            using (SaveOffset())
            {
                return Read("...") && IsWhiteSpace(Read());
            }
        }

        private bool SetToken(Token token)
        {
            EndToken();

            switch (token)
            {
                case Token.ImplicitKey:
                case Token.BlockKeyIndicator:
                case Token.FlowKeyIndicator:
                    InKey = true;
                    break;
                case Token.ValueIndicator:
                    InKey = false;
                    break;
            }

            if (TraceTokens.TraceInfo)
            {
                if (_token == Token.NoOp || _token == token)
                    TraceToken("", token);
                else
                    TraceToken("Change", token);
            }

            _token = token;
            return true;
        }

        private bool DocumentContinue()
        {
            if (ScanIndent() || ScanCommentOrBreak())
                return true;

            char ch = Peek();

            if (Char.IsControl(ch))
            {
                Read();
                return SetToken(Token.Unexpected);
            }

            if (ch == '%')
            {
                ReadLine();
                return SetToken(Token.Directive);
            }

            Expect(YamlExpect.Nothing);
            BlockNodeBegin();
            return true;
        }

        private bool NothingContinue()
        {
            // TODO: Is it really nothing we expect here????

            if (ScanIndent() || ScanSeperation() || ScanCommentOrBreak())
                return true;
            ReadLine();
            AddError("YA116");
            return SetToken(Token.Unexpected);
        }

        /// <summary>
        /// Dispatch to proper "Continue" routine given current state.
        /// </summary>
        /// <returns>dummy</returns>
        [DebuggerStepThrough]
        private bool ContinueScanning()
        {
            switch (State.Expect)
            {
                case YamlExpect.Document:
                    return DocumentContinue();

                case YamlExpect.BlockNode:
                    return BlockNodeContinue();

                case YamlExpect.FlowNode:
                    return FlowNodeContinue();

                case YamlExpect.FlowSeq:
                case YamlExpect.FlowMap:
                    return FlowCollectionContinue();

                case YamlExpect.SingleQuoted:
                    return SingleQuotedContinue();

                case YamlExpect.SinglePrime:
                    return SinglePrime();

                case YamlExpect.DoubleQuoted:
                    return DoubleQuotedContinue();

                case YamlExpect.DoublePrime:
                    return DoublePrime();

                case YamlExpect.DoubleEscapedLineBreak:
                    return DoubleEscapedLineBreakContinue();

                case YamlExpect.FlowPlain:
                    return FlowPlainContinue();

                case YamlExpect.BlockPlain:
                    return BlockPlainContinue();

                case YamlExpect.FoldedText:
                    return FoldedContinue();

                case YamlExpect.Literal:
                    return LiteralContinue();

                case YamlExpect.BlockMap:
                    return BlockMappingContinue();

                case YamlExpect.BlockSeq:
                    return ListContinue();

                case YamlExpect.Nothing:
                    return NothingContinue();

                default:
                    throw new NotImplementedException(
                        String.Format(CultureInfo.InvariantCulture,
                                      "Unexpected style {0}", State.Expect));
            }
        }


        private void SetIndent(int indent)
        {
            int i = indent >= 0 ? indent : ~indent;

            if (TraceStateChanges.TraceWarning && State.Indent > i)
                TraceLine("Warning: incorrect indent in SetIndent()");

            State = new ScanState(State.Expect, (short) indent, TokenStart, InKey);

            if (TraceStateChanges.TraceInfo)
            {
                TraceState("Indent");
            }
        }

        private void Expect(YamlExpect expect)
        {
            Expect(expect, State.RawIndent);
        }

        private void ExpectFlow(YamlExpect expect)
        {
            Expect(expect, State.Indent);
        }

        private void ExpectIndent(YamlExpect expect)
        {
            Expect(expect, MoreIndented(State.Indent));
        }

        /// <summary>
        /// Change processing state without changing indent. Pushes old state on stack if needed.
        /// </summary>
        /// <param name="expect">Next construct to expect.</param>
        /// <param name="indent">Indent level processing</param>
        private void Expect(YamlExpect expect, short indent)
        {
            switch (expect)
            {
                case YamlExpect.Document:
                case YamlExpect.Nothing:
                    break;

                default:
                    if (State.Expect != YamlExpect.BlockNode
                        && State.Expect != YamlExpect.FlowNode)
                    {
                        PushState(State);
                        if (TraceStateChanges.TraceInfo)
                            TraceState(string.Format(CultureInfo.InvariantCulture,
                                                     "Psh({0})", StateDepth - 1));
                    }
                    break;
            }

            State = new ScanState(expect, indent, TokenStart, InKey);

            if (TraceStateChanges.TraceInfo)
            {
                TraceState("Expect");
            }
        }


        private bool PopAndSetToken(Token token)
        {
            if (StateDepth > 0)
            {
                SetToken(token);

                State = PopState();

                if (TraceStateChanges.TraceInfo)
                    TraceState(string.Format(CultureInfo.InvariantCulture,
                                             "Pop({0})", StateDepth));

                return true;
            }

            if (TraceStateChanges.TraceWarning)
                TraceLine("Warning: Attempted to pop empty state stack");

            ScanEOL();
            return SetToken(Token.InconsistentIndent);
        }

        private bool BlockNodeBegin()
        {
            // [125] a block node either consists of a 
            // flow node, or a block-in-block node.   We
            // should be able to tell which one now...
            Expect(YamlExpect.BlockNode);
            if (ScanSeperation())
                return true;
            return BlockNodeContinue();
        }

        private bool BlockNodeContinue()
        {
            if (ScanIndent() || ScanSeperation() || ScanCommentOrBreak())
                return true;

            switch (Peek())
            {
                case '?':
                    return BlockMappingBegin();

                case '|':
                case '>':
                    return BlockScalarBegin();

                case '-':
                    if (IsListIndicator())
                        return ListBegin();
                    break;

                case ':':

                    if (IsValidIndicator())
                    {
                        Read();
                        return BlockMappingBegin();
                    }
                    break;

                case '!': // Tag
                    // In this dumper, a property can procede a
                    // simple isKey, but the BlockMappingBegin must be
                    // emitted before the property.

                    if (IsImplicitKey(BlockPlainBreakCharacters))
                        return BlockMappingBegin();

                    ScanTag();
                    return SetToken(Token.Tag);

                case '&': // Anchor
                    // In this dumper, a property can procede a
                    // simple isKey, but the BlockMappingBegin must be
                    // emitted before the property.

                    if (IsImplicitKey(BlockPlainBreakCharacters))
                        return BlockMappingBegin();

                    FinishAnchorName();
                    return SetToken(Token.Anchor);

                case Unicode.EOS:
                    return PopAndSetToken(Token.Outdent);
            }

            if (IsImplicitKey(BlockPlainBreakCharacters))
                return BlockMappingBegin();

            return CommonNodeContinue();
        }

        private bool ScanProperty()
        {
            if (IsImplicitKey(FlowPlainBreakCharacters))
                return SetToken(Token.ImplicitKey);

            switch (Peek())
            {
                case '!': // Tag
                    return ScanTag();

                case '&':
                    FinishAnchorName();
                    return SetToken(Token.Anchor);

                default:
                    return false;
            }
        }


        protected bool ScanTag()
        {
            Read();
            if (Match('<'))
            {
                FinishUriChars();
                if (!Match('>')) AddError("YA121");
            }
            else
            {
                FinishTagChars();
            }
            return SetToken(Token.Tag);
        }

        private static bool IsAnchorChar(char ch)
        {
            switch (ch)
            {
                case ',':
                case '{':
                case '}':
                case '[':
                case ']':
                case Unicode.Space:
                case Unicode.Tab:
                case Unicode.EOS:
                    return false;

                default:
                    if (Char.IsControl(ch)) return false;
                    return true;
            }
        }


        // We leave deter
        protected void FinishAnchorName()
        {
            Read();
            BeginScan();
            if (!IsAnchorChar(Peek()))
                AddError("YA310");

            while (IsAnchorChar(Peek()))
            {
                Read();
            }
            EndToken();
            return;
        }

        private static bool IsHexDigit(char c)
        {
            return (c >= '0' && c <= '9')
                   || (c >= 'a' && c <= 'f')
                   || (c >= 'A' && c <= 'F');
        }

        private static bool IsWordChar(char c)
        {
            return (c >= '0' && c <= '9')
                   || (c >= 'A' && c <= 'Z')
                   || (c >= 'a' && c <= 'z')
                   || (c == '-');
        }

        [SuppressMessage(
            "Microsoft.Maintainability",
            "CA1502:AvoidExcessiveComplexity",
            Justification = "Scanning sometimes requires complexity"
            )]
        protected void FinishUriChars()
        {
            while (true)
            {
                char c = Read();

                if (IsWordChar(c)) continue;
                if (c == '%')
                {
                    // Must have 2 hex digits following it
                    if (!IsHexDigit(Read())) AddError("YA311");
                    if (!IsHexDigit(Read())) AddError("YA311");
                    continue;
                }
                switch (c)
                {
                    case '#':
                    case ';':
                    case '/':
                    case '?':
                    case ':':
                    case '@':
                    case '&':
                    case '=':
                    case '+':
                    case '$':
                    case ',':
                    case '_':
                    case '.':
                    case '!':
                    case '~':
                    case '*':
                    case '(':
                    case ')':
                    case '[':
                    case ']':
                    case '\'':
                        continue;
                }
                break;
            }

            Backup(1);
            EndToken();
            return;
        }

        [SuppressMessage(
            "Microsoft.Maintainability",
            "CA1502:AvoidExcessiveComplexity",
            Justification = "Its just not that bad..."
            )]
        private static bool IsTagChar(char c)
        {
            if (IsWordChar(c)) return true;
            switch (c)
            {
                case '#':
                case ';':
                case '/':
                case '?':
                case ':':
                case '@':
                case '&':
                case '=':
                case '+':
                case '$':
                case '_':
                case '.':
                case '~':
                case '*':
                case '(':
                case ')':
                case '\'':
                case '%':
                case '!': // Leaves further processing of tag to higher levels
                    return true;
            }
            return false;
        }

        protected void FinishTagChars()
        {
            while (IsTagChar(Peek()))
            {
                Read();
            }
            EndToken();
            return;
        }


        private bool ListBegin()
        {
            Expect(YamlExpect.BlockSeq, CurrentIndent); // State: in a block sequence
            Read(); // Skip over the "-"
            ExpectIndent(YamlExpect.BlockNode); // Get ready to read a node
            return SetToken(Token.BlockSeqBegin);
        }

        private bool ListContinue()
        {
            if (ScanIndent() || ScanSeperation() || ScanCommentOrBreak())
                return true;

            if (State.Indent + 1 != TokenStart.Column)
            {
                ReadLine();
                return SetToken(Token.InconsistentIndent);
            }

            switch (Peek())
            {
                case '-':
                    if (IsListIndicator())
                    {
                        Read(); // Skip over the '-'
                        ExpectIndent(YamlExpect.BlockNode);
                        return SetToken(Token.BlockSeqNext);
                    }
                    break;
            }
            return PopAndSetToken(Token.Outdent);
        }

        private bool ScanUnexpected()
        {
            AddError("YA101", DisplayChar(Read()));
            return SetToken(Token.Unexpected);
        }

        private bool BlockMappingBegin()
        {
            Expect(YamlExpect.BlockMap, CurrentIndent);
            return SetToken(Token.BlockMappingBegin);
        }

        private bool BlockMappingContinue()
        {
            if (ScanIndent()) return true;
            if (_lastToken != Token.Outdent && ScanSeperation())
                return true;

            switch (Peek())
            {
                case Unicode.Space:
                    ScanSeperation();
                    return SetToken(Token.InconsistentIndent);

                case '?':
                    Read(); // Consume the ?
                    if (State.Indent + 1 != TokenStart.Column)
                    {
                        ReadLine();
                        AddError("YA118", State.Indent, TokenStart.Column);
                        return SetToken(Token.Unexpected);
                    }
                    SetToken(Token.BlockKeyIndicator);
                    ExpectIndent(YamlExpect.BlockNode);
                    return true;

                case ':':
                    if (IsValidIndicator())
                    {
                        Read();
                        SetToken(Token.ValueIndicator);
                        ExpectIndent(YamlExpect.BlockNode);
                        return true;
                    }
                    break;

                case '-':
                    if (IsListIndicator())
                    {
                        if (TraceTokens.TraceInfo)
                            TraceLine("Note: special case sequence considered embedded in mapping");
                        return ListBegin();
                    }
                    break;
            }

            if (ScanProperty())
                return true;

            return BlockNodeBegin();
        }

        private bool BlockScalarBegin()
        {
            char indicator = Read();

            _chomping = ' '; // Set default chomping
            ScanChomping(); // Scan for a chomping indicator
            short indent = ScanIndentationIndicator();
            ScanChomping(); // Scan for a chomping indicator (again)
            AdvanceToCommentOrEnd();

            switch (indicator)
            {
                case '|':
                    SetToken(Token.Literal);
                    Expect(YamlExpect.Literal, indent);
                    break;
                case '>':
                    SetToken(Token.Folded);
                    Expect(YamlExpect.FoldedText, indent);
                    break;
            }
            return true;
        }

        private short ScanIndentationIndicator()
        {
            char ch = Peek();
            int baseIndent = IsIndentPending ? State.Indent - 1 : State.Indent;

            if (ch >= '1' && ch <= '9')
                return (short) Math.Max(0, Read() - '0' + baseIndent);
            else
                return (MoreIndented(baseIndent));
        }

        /// <summary>
        /// Scan for a chomping indicator.  Update _chomping if found;
        /// </summary>
        /// <returns>True if indicator found.</returns>
        private bool ScanChomping()
        {
            if (_chomping == ' ') // Don't bother if a chomping indicator has already been found
            {
                if (Read('+'))
                    _chomping = '+';
                else if (Read('-'))
                    _chomping = '-';
                return _chomping != ' ';
            }
            return false;
        }

        private bool FoldedContinue()
        {
            if (_lastToken == Token.Folded && Peek() == '#')
                return ScanCommentOrBreak();

            if (ScanIndent())
                return true;

            if (State.IsIndentPending)
            {
                SkipSpaces();
                SetToken(Token.IndentSpaces);
                SetIndent(TokenSpan.End.Column - 1);
                return true;
            }

            switch (Read())
            {
                case Unicode.Space:
                case Unicode.Tab:
                    SkipSpacesAndTabs();
                    return SetToken(Token.LeadingWhiteSpace);
            }

            ReadLine();
            return SetToken(Token.TextContent);
        }

        private bool LiteralContinue()
        {
            if (_lastToken == Token.Literal && Peek() == '#')
                return ScanCommentOrBreak();

            if (ScanIndent()) return true;

            if (State.IsIndentPending)
            {
                SkipSpaces();
                SetToken(Token.IndentSpaces);
                SetIndent(TokenSpan.End.Column - 1);
                return true;
            }
            ReadLine();
            return SetToken(Token.TextContent);
        }

        private bool FlowNodeContinue()
        {
            return CommonNodeContinue();
        }

        private bool CommonNodeContinue()
        {
            if (ScanIndent() || ScanSeperation() || ScanCommentOrBreak())
                return true;

            if (ScanProperty())
                return true;

            switch (Peek())
            {
                case '*': // Alias
                    FinishAnchorName();
                    if (State.Expect == YamlExpect.BlockNode
                        || State.Expect == YamlExpect.FlowNode)
                        return PopAndSetToken(Token.Alias);
                    else
                        return SetToken(Token.Alias);

                case '[':
                    Read();
                    SetToken(Token.OpenBracket);
                    ExpectFlow(YamlExpect.FlowSeq);
                    return true;

                case '{':
                    Read();
                    SetToken(Token.OpenBrace);
                    ExpectFlow(YamlExpect.FlowMap);
                    return true;

                case '?':
                    if (Peek() == Unicode.Space)
                    {
                        Read();
                        Read();
                        SetToken(Token.FlowKeyIndicator);
                        return true;
                    }
                    break;

                case ':':
                    if (IsValidIndicator())
                    {
                        Read();
                        return SetToken(Token.ValueIndicator);
                    }
                    break;

                case Unicode.DoubleQuote:
                    Read();
                    ExpectFlow(YamlExpect.DoubleQuoted);
                    return SetToken(Token.DoubleQuote);

                case Unicode.SingleQuote:
                    Read();
                    ExpectFlow(YamlExpect.SingleQuoted);
                    return SetToken(Token.SingleQuote);
            }

            return PlainBegin();
        }

        private bool FlowCollectionContinue()
        {
            // ScanState thisState = State; // Dead?

            if (ScanIndent())
            {
                if (Token == Token.Outdent)
                {
                    //AddError(thisState.StartLocation, "YA113");
                }
                return true;
            }

            if (ScanSeperation()) return true;

            switch (Peek())
            {
                case ']':
                    Read();
                    return PopAndSetToken(Token.CloseBracket);

                case '}':
                    Read();
                    return PopAndSetToken(Token.CloseBrace);

                case ':':
                    if (IsValidIndicator())
                    {
                        Read();
                        return SetToken(Token.ValueIndicator);
                    }
                    break;

                case ',':
                    Read();
                    return SetToken(Token.Comma);

                case '?':
                    Read();
                    SetToken(Token.FlowKeyIndicator);
                    ExpectFlow(YamlExpect.FlowNode);
                    return true;
            }
            return FlowNodeContinue();
        }

        private bool PlainBegin()
        {
            SetToken(Token.PlainScalar);
            ExpectFlow(State.IsBlockContext
                           ? YamlExpect.BlockPlain
                           : YamlExpect.FlowPlain);
            return true;
        }

        private bool BlockPlainContinue()
        {
            if (PeekToken("#"))
                return PopAndSetToken(Token.PlainEnd);

            if (ScanIndent() || ScanSeperation())
            {
                if (Token == Token.Outdent)
                    SetToken(Token.PlainEnd);
                return true;
            }

            while (true)
            {
                switch (Peek())
                {
                    case '#':
                        return ScanCommentOrBreak();

                    case ':':
                        if (State.InKey && IsValidIndicator())
                        {
                            return PopAndSetToken(Token.PlainEnd);
                        }
                        break;
                }
                Read();
                SkipPlain(BlockPlainBreakCharacters);
                if (Peek() == Unicode.Tab)
                {
                    // Tabs are permitted in plain scalars only 
                    // as part of ignored whitespace at BOL & EOL
                    Read();
                    int length = ReadSpacesAndTabs();
                    if (!(Peek() == '#' || IsBreak(Peek())))
                    {
                        Backup(length);
                        return ScanUnexpected();
                    }
                    Backup(length + 1);
                }
                BackOverSpacesOnly();
                return SetToken(Token.TextContent);
            }
        }

        private bool SingleQuotedContinue()
        {
            if (ScanIndent()) return true;

            if (_lastToken == Token.IndentSpaces || CurrentIndent == 0)
                if (ScanSeperation())
                    return true;

            switch (Read())
            {
                default:
                    AdvanceTo(SingleQuoteBreakCharacters);
                    if (IsBreak(Peek()))
                        BackOverSpacesAndTabs();
                    return SetToken(Token.TextContent);

                case Unicode.SingleQuote:
                    if (Peek() == Unicode.SingleQuote)
                    {
                        Expect(YamlExpect.SinglePrime);
                        return SetToken(Token.EscapeWhiteSpace);
                    }
                    Backup(1);
                    UnfoldFlowTextToken();
                    Read(Unicode.SingleQuote);
                    EndToken();
                    return PopAndSetToken(Token.SingleQuote);

                case Unicode.Tab:
                case Unicode.Space:
                    ReadSpacesAndTabs();

                    // Ignore trailing whitespace
                    if (IsBreak(Peek()))
                        return ScanEOL();

                    // Otherwise tread as normal
                    AdvanceTo(SingleQuoteBreakCharacters);
                    return SetToken(Token.TextContent);

                case Unicode.CR:
                case Unicode.LF:
                case Unicode.NUL:
                    Backup(1);
                    return ScanEOL();
            }
        }

        private bool SinglePrime()
        {
            // Special logic applied for a literal quote in a single quoted scalar:
            TokenText = Read().ToString();
            return PopAndSetToken(Token.TextContent);
        }

        private bool ScanEOL()
        {
            switch (Peek())
            {
                case Unicode.EOS:
                    //if (Depth > 0)
                    //{
                    //    switch (_lastToken)
                    //    {
                    //        case Token.Newline:
                    //        case Token.SpecificEndOfLine:
                    //        case Token.Outdent:
                    //            return PopAndSetToken(Token.Outdent);
                    //    }
                    //    return SetToken(Token.Newline);
                    //}
                    //else
                    return SetToken(Token.EndOfStream);

                case Unicode.CR:
                case Unicode.LF:
                    break;

                default:
                    return false;
            }

            SkipEOL();
            return SetToken(Token.Newline);
        }

        // Used for higher-level error reporting/recovery
        public void SkipToEOL()
        {
            ReadLine();
            EndToken();
        }

        private void SkipSingleQuoted()
        {
            Read(); // Assumed to be leading quote
            while (true)
            {
                AdvanceTo(SingleQuoteBreakCharacters);
                switch (Peek())
                {
                    case Unicode.SingleQuote:
                        Read();
                        if (Peek() == Unicode.SingleQuote)
                        {
                            Read(); // Skip doubled quote
                            break; // And keep going
                        }
                        return;

                    default:
                        return;
                }
            }
        }

        private void SkipDoubleQuoted()
        {
            Read(); // Assumed to be leading quote
            while (true)
            {
                AdvanceTo(DoubleQuoteBreakCharacters);
                switch (Peek())
                {
                    case Unicode.DoubleQuote:
                        Read();
                        return;

                    case Unicode.Backslash:
                        // In this dumper we only need to check for escaped quote
                        Read();
                        if (Peek() == Unicode.DoubleQuote)
                            Read();
                        break;

                    default:
                        return;
                }
            }
        }

        private bool UnfoldFlowTextToken()
        {
            EndToken();
            return SetToken(Token.TextContent);
        }


        [SuppressMessage(
            "Microsoft.Maintainability",
            "CA1502:AvoidExcessiveComplexity",
            Justification = "Scanning sometimes requires complexity"
            )]
        private bool DoubleQuotedContinue()
        {
            if (ScanIndent()) return true;

            if (_lastToken == Token.IndentSpaces || CurrentIndent == 0)
                if (ScanSeperation())
                    return true;

            while (true)
            {
                switch (Peek())
                {
                    default:
                        AdvanceTo(DoubleQuoteBreakCharacters);
                        if (IsBreak(Peek())) // At EOL...
                            BackOverSpacesAndTabs(); // ignore trailing whitespace.
                        return SetToken(Token.TextContent);

                    case Unicode.Tab:
                    case Unicode.Space:
                        ReadSpacesAndTabs();

                        // Ignore trailing whitespace
                        if (IsBreak(Peek())) return ScanEOL();

                        // Otherwise tread as normal
                        goto default;

                    case Unicode.DoubleQuote:
                        UnfoldFlowTextToken();
                        Read();
                        EndToken();
                        return PopAndSetToken(Token.DoubleQuote);

                    case Unicode.Backslash:
                        Read();
                        switch (Peek())
                        {
                                // These escapes get slightly special treatment
                                // designed to make the following character show 
                                // up with content coloring in the editor...

                            case Unicode.Space:
                            case Unicode.Tab:
                            case Unicode.DoubleQuote:
                            case Unicode.Backslash:
                                Expect(YamlExpect.DoublePrime);
                                return SetToken(Token.EscapeWhiteSpace);


                            case '0':
                                return Insert((char) 0); // Null
                            case 'a':
                                return Insert('\a'); // Alert (bell)
                            case 'b':
                                return Insert('\b'); // Backspace
                            case 't':
                                return Insert('\t'); // Tab character 
                            case 'n':
                                return Insert('\n'); // Newline (linefeed)
                            case 'v':
                                return Insert('\v'); // Vertical tab
                            case 'f':
                                return Insert('\f'); // Form feed
                            case 'r':
                                return Insert('\r'); // Return 
                            case 'e':
                                return Insert(Unicode.Escape);
                            case 'N':
                                return Insert('\x85');
                            case '_':
                                return Insert((char) 0xa0); // Non breaking space
                            case 'L':
                                return Insert('\u2028'); // Line separator
                            case 'P':
                                return Insert('\u2029'); // Paragraph separator

                            case 'x':
                                return ScanUnicodeHex(2);
                            case 'u':
                                return ScanUnicodeHex(4);
                            case 'U':
                                return ScanUnicodeHex(8);

                            case Unicode.CR:
                            case Unicode.LF:
                                SkipEOL();
                                Expect(YamlExpect.DoubleEscapedLineBreak);
                                return SetToken(Token.EscapedLineBreak);
                        }

                        // Unexpected
                        Read();
                        AddError("YA105");
                        return SetToken(Token.Escape);
                }
            }
        }

        private bool DoubleEscapedLineBreakContinue()
        {
            // Special logic applied after an escaped 
            // line break in a double quoted scalar:

            // At the first printable data, we suppress 
            // any counted linebreaks and revert to normal 
            // double quoted processing.  [135]

            PopAndSetToken(Token.NoOp);
            return DoubleQuotedContinue();
        }

        private bool DoublePrime()
        {
            // Special logic to make an escaped character part of content
            TokenText = Read().ToString();
            return PopAndSetToken(Token.TextContent);
        }


        private bool Insert(char c)
        {
            Read();
            TokenText = c.ToString();
            return SetToken(Token.Escape);
        }

        private bool ScanUnicodeHex(int digits)
        {
            Read(); // skip prefix char

            Int32 value = 0;
            for (int i = 0; i < digits; i++)
            {
                Char ch = Char.ToUpper(Read(), CultureInfo.InvariantCulture);
                value *= 16;
                if (ch >= '0' && ch <= '9')
                    value += ch - '0';
                else if (ch >= 'A' && ch <= 'F')
                    value += ch - 'A' + 10;
                else
                {
                    while (++i < digits) Read();
                    AddError("YA106", digits);
                    return SetToken(Token.Escape);
                }
            }
            if (value < 0 || value > 0x10ffff)
                AddError("YA107");
            else if (Char.IsSurrogate((char) value))
                AddError("YA108");
            else
                TokenText = Char.ConvertFromUtf32(value);
            return SetToken(Token.Escape);
        }

        private bool ScanCommentOrBreak()
        {
            if (Peek() == Unicode.EOS)
            {
                if (_incrementalMode) return SetToken(Token.EndOfStream);
                return PopAndSetToken(Token.Outdent);
            }
            if (Read('#'))
            {
                ReadLine(); // Skip rest of line
                return SetToken(Token.Comment); // And return the comment token
            }
            return ScanEOL();
        }

        // The plain style uses no identifying indicators, and is therefore the 
        // most limited and most dumper sensitive scalar style. Plain scalars 
        // can never contain any tab characters. They also must not contain the 
        // ': ' and ' #' character sequences as these combinations cause ambiguity 
        // with isKey: value pairs and comments. Inside flow collections, plain scalars 
        // are further restricted to avoid containing the '[', ']', '{', '}' and ',' 
        // characters as these would cause ambiguity with the flow collection structure 
        // (hence the need for the flow-in dumper and the flow-out dumper).
        // Scan a single-line plain scaler per ns-plain-single(c)

        private void SkipPlain(char[] breakCharacters)
        {
            while (true)
            {
                AdvanceTo(breakCharacters);

                switch (Peek())
                {
                    default:
                        return;

                    case '#': // Possible comment, but must be proceded by space
                        if (PreviousChar == Unicode.Space)
                            return;
                        Read();
                        break;

                    case ':': // Possible isKey, check
                        if (IsValidIndicator())
                            return;
                        Read();
                        break;
                }
            }
        }

        private bool IsValidIndicator()
        {
            char lookahead = Peek(1);

            if (!State.IsBlockContext)
            {
                // Yaml 1.2 in flow dumper: JSON style
                // No following whitespace requred under strict conditions.
                switch (_lastToken)
                {
                    case Token.CloseBrace:
                    case Token.CloseBracket:
                    case Token.DoubleQuote:
                    case Token.SingleQuote:
                        return true;
                }
                switch (lookahead)
                {
                    case '}':
                    case ']':
                    case ',':
                        return true;
                }
            }
            return IsWhiteSpace(lookahead);
        }

        private bool FlowPlainContinue()
        {
            if (PeekToken("#"))
                return SetToken(Token.PlainEnd);

            if (ScanIndent() || ScanSeperation())
            {
                if (Token == Token.Outdent)
                    SetToken(Token.PlainEnd);
                return true;
            }

            while (true)
            {
                switch (Peek())
                {
                    case '#':
                        //
                        // '#' isn't one of the plain break characters, so 
                        // to get here, it must have been preceded by one,
                        // making this a comment.
                        //
                        return ScanCommentOrBreak();

                    case '[':
                    case ']':
                    case '{':
                    case '}':
                    case ',':
                        // In a flow collection, these end a plain scalar
                        return PopAndSetToken(Token.PlainEnd);

                    case ':':
                        //    
                        // ':' must be followed by some whitespace to end a plain
                        //
                        if (IsValidIndicator())
                        {
                            return PopAndSetToken(Token.PlainEnd);
                        }
                        break;
                }
                //
                // Skip to the next posible break position and return text
                //
                Read();
                SkipPlain(FlowPlainBreakCharacters);
                if (Peek() == Unicode.Tab)
                {
                    // Tabs are permitted in plain scalars only 
                    // as part of ignored whitespace at BOL & EOL
                    Read();
                    int length = ReadSpacesAndTabs();
                    if (!(Peek() == '#' || IsBreak(Peek())))
                    {
                        Backup(length);
                        return ScanUnexpected();
                    }
                    Backup(length - 1);
                }
                BackOverSpacesOnly();
                return SetToken(Token.TextContent);
            }
        }

        private void AdvanceToCommentOrEnd()
        {
            char ch = Unicode.EOS;
            while (true)
            {
                char previous = ch;
                ch = Read();
                if (IsBreak(ch) || ch == '#' && previous == Unicode.Space)
                {
                    if (ch == '#')
                        Backup(1);
                    return;
                }
            }
        }

        /// <summary>
        /// Detects if a colon follows the current node, making it an implicit isKey
        /// </summary>
        /// <param name="breakCharacters">Characters to skip</param>
        [SuppressMessage(
            "Microsoft.Maintainability",
            "CA1502:AvoidExcessiveComplexity",
            Justification = "Scanning sometimes requires complexity"
            )]
        protected bool IsImplicitKey(char[] breakCharacters)
        {
            // Avoid generating implicit isKey indication multiple times...

            switch (_lastToken)
            {
                case Token.ImplicitKey:
                case Token.FlowKeyIndicator:
                case Token.Tag:
                case Token.Anchor:
                case Token.Alias:
                    return false;
            }

            int endOffset = Offset + 1024;

            using (SaveOffset())
            {
                // This is the fast little state machine that runs forward through
                // the buffer trying to determine if the current node looks like a isKey.

                int flowDepth = 0;
                while (endOffset > Offset)
                {
                    switch (Peek())
                    {
                        case Unicode.CR:
                        case Unicode.LF:
                        case Unicode.EOS:
                            return false;

                        case '-':
                            if (IsListIndicator() && Peek() == Unicode.Space)
                                return false;
                            goto default;

                        case '?':
                        case '#':
                        case '|':
                        case '@':
                        case '`':
                            return false;

                        case ':':
                            return false; // Is this a special case?

                        case ',':
                            if (flowDepth == 0)
                                return false;
                            Read();
                            continue;


                        case Unicode.Tab:
                            Read();
                            continue;

                        case '!':
                            Read();
                            if (Peek() == '<')
                            {
                                Read();
                                FinishUriChars();
                                if (!Match('>')) AddError("YA121");
                            }
                            else
                                FinishTagChars();
                            SkipSpaces();
                            continue;

                        case '*':
                        case '&':
                            FinishTagChars();
                            SkipSpaces();
                            continue;

                        case '[':
                        case '{':
                            flowDepth++;
                            Read();
                            continue;

                        case ']':
                        case '}':
                            if (--flowDepth < 0)
                                return false;
                            Read();
                            SkipSpaces();
                            if (flowDepth == 0)
                                return Peek() == ':';
                            continue;

                        default:
                            // Determine if it's a plain followed by a colon
                            SkipPlain(flowDepth > 0
                                          ? FlowPlainBreakCharacters
                                          : breakCharacters);
                            SkipSpaces();
                            if (Peek() == ':')
                            {
                                if (flowDepth == 0)
                                    return true;
                                Read();
                                SkipSpaces();
                            }
                            continue;

                        case Unicode.SingleQuote:
                            SkipSingleQuoted();
                            SkipSpaces();
                            if (Peek() == ':')
                            {
                                if (flowDepth == 0)
                                    return true;
                                Read();
                                SkipSpaces();
                            }
                            continue;

                        case Unicode.DoubleQuote:
                            SkipDoubleQuoted();
                            SkipSpaces();
                            if (Peek() == ':')
                            {
                                if (flowDepth == 0)
                                    return true;
                                Read();
                                SkipSpaces();
                            }
                            continue;
                    }
                }
                return false;
            }
        }

        #region Nested type: ScanState

        /// <summary>
        /// All inter-line state must fit in this class to make the VS tokenizer work right.
        /// </summary>
        protected struct ScanState : IEquatable<ScanState>
        {
            private const string _indentNotEstablished = "Current indent not established";

            public static readonly ScanState Default = new ScanState(
                YamlExpect.Document, -1, SourceLocation.MinValue, false);

            private readonly short _indent;
            private readonly SourceLocation _startLocation;

            internal ScanState(YamlExpect expect, short indent, SourceLocation location, bool inKey)
                : this()
            {
                Expect = expect;
                _indent = indent;
                _startLocation = location;
                InKey = inKey;
            }

            public YamlExpect Expect { get; private set; }
            public bool InKey { get; private set; }


            // Indent is isKey to Yaml's block structure, very Python like.
            //
            // Non-negative indent values mean we're expecting exactly n leadingSpaces of indent.
            // _indent value of -1 is a special starting state assumed in the YAML productions, it
            // always gets incremented to at least 0 before any real text is processed.
            //
            // There are cases in YAML where an new indent must be established (greater than the 
            // current one) yet that actual value cannot be determined until the next line is 
            // proceesed.  This state is represented internally by _indent values <-1, by using ones
            // complement notation of the minimum expected value.   
            //

            public bool IsIndentPending
            {
                get { return _indent <= -1; }
            }

            public short Indent
            {
                get { return (short) ((IsIndentPending) ? ~_indent : _indent); }
            }


            public bool InQuotes
            {
                get
                {
                    switch (Expect)
                    {
                        case YamlExpect.DoubleQuoted:
                        case YamlExpect.DoublePrime:
                        case YamlExpect.DoubleEscapedLineBreak:
                        case YamlExpect.SingleQuoted:
                        case YamlExpect.SinglePrime:
                            return true;
                    }
                    return false;
                }
            }


            internal short RawIndent
            {
                get { return _indent; }
            }

            // Note that the StartLocation is _not_ included in equality comparison
            // this is by design, allowing them to be shared on the scanner seekable 
            // stack.   We need SourceLocation primarily for error reporting, which
            // happens doesn't happen on the same scanner.

            public SourceLocation StartLocation
            {
                get { return _startLocation; }
            }


            internal bool IsBlockContext
            {
                get
                {
                    switch (Expect)
                    {
                        case YamlExpect.BlockMap:
                        case YamlExpect.BlockNode:
                        case YamlExpect.BlockPlain:
                        case YamlExpect.BlockSeq:
                        case YamlExpect.Document:
                            return true;
                    }
                    return false;
                }
            }

            internal bool IsPlainContext
            {
                get
                {
                    switch (Expect)
                    {
                        case YamlExpect.FlowPlain:
                        case YamlExpect.BlockPlain:
                            return true;
                    }
                    return false;
                }
            }

            #region IEquatable<ScanState> Members

            public bool Equals(ScanState other)
            {
                if (other == null)
                    return false;
                return this == other;
            }

            #endregion

            public override bool Equals(object obj)
            {
                return obj is ScanState && ((ScanState) obj) == this;
            }

            public override int GetHashCode()
            {
                return Indent.GetHashCode() ^ Expect.GetHashCode() ^ (InKey ? 0x280000 : 0);
            }

            public static bool operator ==(ScanState one, ScanState other)
            {
                return one._indent == other._indent
                       && one.Expect == other.Expect
                       && one.InKey == other.InKey;
            }

            public static bool operator !=(ScanState one, ScanState other)
            {
                return !(one == other);
            }

            public override string ToString()
            {
                return string.Format(CultureInfo.InvariantCulture, "{0} \t{1}{2}",
                                     Expect, Indent, IsIndentPending ? "+" : " ");
            }
        }

        #endregion

        #region Nested type: YamlExpect

        protected enum YamlExpect
        {
            Document, // Inital state
            BlockNode,
            BlockSeq,
            BlockMap, // Block constructs
            FlowNode,
            FlowSeq,
            FlowMap, // Flow constructs
            SingleQuoted,
            DoubleQuoted, // Quoted constructs
            FlowPlain,
            BlockPlain, // Plain Scalars
            Literal,
            FoldedText,
            DoublePrime,
            SinglePrime,
            DoubleEscapedLineBreak, // CookScalar content
            Nothing // Terminal state
        };

        #endregion
    }

    [CLSCompliant(false)]
    [ContractVerification(false)] // TODO Remove this whe static verification issues addressed
    public class YamlScanner : AbstractYamlScanner
    {
        // The plain YamlScanner uses a plain Stack of states

        private readonly Stack<ScanState> _stack = new Stack<ScanState>();

        protected override int StateDepth
        {
            get { return _stack.Count; }
        }

        protected override void PushState(ScanState state)
        {
            _stack.Push(state);
        }

        protected override ScanState PopState()
        {
            return _stack.Pop();
        }
    }

    /// <summary>
    /// YamlScanner with Seek/Tell capability for fast colorizing in an editor
    /// </summary>
    [CLSCompliant(false)]
    [ContractVerification(false)] // TODO Remove this whe static verification issues addressed
    public class SeekableYamlScanner : AbstractYamlScanner
    {
        private readonly MnemonicStack<ScanState> _stack = new MnemonicStack<ScanState>();

        protected override int StateDepth
        {
            get { return _stack.Count; }
        }

        protected override void PushState(ScanState state)
        {
            _stack.Push(state);
        }

        protected override ScanState PopState()
        {
            return _stack.Pop();
        }

        /// <summary>
        /// Memorizes the current scanning state for later retrieval.
        /// </summary>
        /// <returns>A memento of the state, usable with SeekState() on the same scanner instance.</returns>
        public Int32 TellState()
        {
            return _stack.Memorize(State);
        }

        /// <summary>
        /// Enable incremental out-of-order scanning from a previous state.
        /// </summary>
        /// <param name="text"></param>
        /// <param name="mnemonic">Zero, or a memento generated by TellState() on the same instance.</param>
        /// <param name="location"></param>
        /// <remarks>
        /// <para>
        /// Useful for incremental scanning, such as when used as a Visual Studio
        /// colorizer, which operates line-at-a-time and often out of sequential order.   
        /// This requires complementary use of SeekState() and TellState()
        /// </para>
        /// <para>
        /// Passing a value of zero to SeekState, sets the scanner to its default initial
        /// incremental state.   This must be done before any scanning has started to enable
        /// incremental sink.  
        /// 
        /// Individual lines are scanned, after which TellState() should be called
        /// and the result saved.   
        /// </para>
        /// </remarks>
        public void SeekState(SourceLocation location, string text, Int32 mnemonic)
        {
            location = SeekState(location, text);

            if (mnemonic == 0)
            {
                _stack.Clear();
                _stack.TrimExcess();
                State = ScanState.Default;
            }
            else
            {
                State = _stack.Recall((short) mnemonic);
            }
        }
    }
}